Advanced Lane Finding Project

The goals / steps of this project are the following:

  • Compute the camera calibration matrix and distortion coefficients given a set of chessboard images.
  • Apply a distortion correction to raw images.
  • Use color transforms, gradients, etc., to create a thresholded binary image.
  • Apply a perspective transform to rectify binary image ("birds-eye view").
  • Detect lane pixels and fit to find the lane boundary.
  • Determine the curvature of the lane and vehicle position with respect to center.
  • Warp the detected lane boundaries back onto the original image.
  • Output visual display of the lane boundaries and numerical estimation of lane curvature and vehicle position.

1 Compute the camera calibration using chessboard images

Please reset the whole project file in your favourite path as the cd path I chose for my own local computer.

In [1]:
%cd /Users/libo/Documents/Test/self_driving/project2_advanced_lane_finding/CarND-Advanced-Lane-Lines
/Users/libo/Documents/Test/self_driving/project2_advanced_lane_finding/CarND-Advanced-Lane-Lines
In [2]:
# Define working path:
path = '/Users/libo/Documents/Test/self_driving/project2_advanced_lane_finding/CarND-Advanced-Lane-Lines'
tst_input = '/test_images/'
output = '/output_images/'
output_v = '/output_videos/'
output_color = '/output_images/color_transform/'
output_chessboard = '/output_images/chessboard/'
output_gradient = '/output_images/gradient_transform/'
output_combined = '/output_images/combined_transform/'
output_cali = '/camera_cal_output/'
cali_input = '/camera_cal/'
In [3]:
import numpy as np
import cv2
import glob
import matplotlib.pyplot as plt
import os
import pandas as pd
import pickle
from ipywidgets import interact, interactive, fixed
from moviepy.editor import VideoFileClip
from IPython.display import HTML
%matplotlib inline

# prepare object points, like (0,0,0), (1,0,0), (2,0,0) ....,(6,5,0)
objp = np.zeros((6*9,3), np.float32)
objp[:,:2] = np.mgrid[0:9,0:6].T.reshape(-1,2)

# Arrays to store object points and image points from all the images.
objpoints = [] # 3d points in real world space
imgpoints = [] # 2d points in image plane.

# termination criteria
criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.001)

# Make a list of calibration images
images = glob.glob('camera_cal/calibration*.jpg')


# Step through the list and search for chessboard corners
for fname in images:
    img = cv2.imread(fname)
    gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)

    # Find the chessboard corners
    ret, corners = cv2.findChessboardCorners(gray, (9,6),None)

    # If found, add object points, image points
    if ret == True:
        objpoints.append(objp)
        imgpoints.append(corners)

        # Draw and display the corners
        img = cv2.drawChessboardCorners(img, (9,6), corners, ret)
#         cv2.imshow('img',img)
#         cv2.waitKey(500)
In [4]:
np.shape(objpoints)
Out[4]:
(17, 54, 3)
In [5]:
np.shape(imgpoints)
Out[5]:
(17, 54, 1, 2)
In [6]:
def read_all_imgs(path , color=cv2.IMREAD_COLOR):

    images = []
    filenames = []
    
    filelist = os.listdir(path)
    
    for file in filelist:
        
        try:
            img = cv2.imread(path+file)
        except:
            img = None
        
        if img is not None:
            img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
            images.append(img)
            filenames.append(file)
    
    return images, filenames
In [7]:
images, filenames = read_all_imgs(path+cali_input)
In [8]:
# We know that the number of chessboard image is 20, 
# so we can give an 5*4 array of pictures enlisting all the pictures,
# some of these images do not appear because 
# the specified number of chessboard corners were not found.
fig, axs = plt.subplots(5, 4, figsize = (16, 11))
fig.subplots_adjust(hspace = .2, wspace = .001)
axs = axs.ravel()

# Step through the list and search for the chessboard corners

for i, fname in enumerate(filenames):
    img = cv2.imread(path+cali_input+fname)
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    # Find the chessboard corners
    ret, corners = cv2.findChessboardCorners(gray, (9, 6), None)
    
    # If found, add object points, image points
    if ret == True:
        objpoints.append(objp)
        # Once we find the corners, we can increase their accuracy using cv2.cornerSubPix().
        # http://opencv-python-tutroals.readthedocs.io/en/latest/py_tutorials/py_calib3d/py_calibration/py_calibration.html
        corners2 = cv2.cornerSubPix(gray, corners, (11,11), (-1,-1),criteria)
        imgpoints.append(corners2)
        
        # Draw and display the corners
        img = cv2.drawChessboardCorners(img, (9,6), corners, ret)
        cv2.imwrite(path + output_chessboard +fname, img)
        axs[i].axis('off')
        axs[i].imshow(img)

2 Apply a distortion correction to raw images

This is the distortion correction to the raw crossboard image

In [9]:
filenames.sort()
In [10]:
# Test undistortion on an image
img = cv2.imread(path + cali_input +filenames[0])
img_size = (img.shape[1], img.shape[0])

# Do camera calibration given object points and image points
ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera(objpoints, imgpoints, img_size,None,None)
dst = cv2.undistort(img, mtx, dist, None, mtx)

# Save the camera calibration result for later use (we won't worry about rvecs / tvecs)
dist_pickle = {}
dist_pickle["mtx"] = mtx
dist_pickle["dist"] = dist
pickle.dump( dist_pickle, open( "calibration.p", "wb" ) )
#dst = cv2.cvtColor(dst, cv2.COLOR_BGR2RGB)
# Visualize undistortion
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(20,10))
f.subplots_adjust(hspace = .2, wspace=.05)
ax1.imshow(img)
ax1.set_title('Original Image', fontsize=30)
ax2.imshow(dst)
ax2.set_title('Undistorted Image', fontsize=30)
print('Comparison between the original image and the undistorted image')
Comparison between the original image and the undistorted image

This is the correction distortion of the sample raw lane image

In [11]:
images, filenames = read_all_imgs(path+tst_input)
In [12]:
filenames.sort()
filenames
Out[12]:
['straight_lines1.jpg',
 'straight_lines2.jpg',
 'test1.jpg',
 'test2.jpg',
 'test3.jpg',
 'test4.jpg',
 'test5.jpg',
 'test6.jpg']
In [13]:
# Choose an image to demonstrate each step of the pipeline
exampleImg = cv2.imread(path+tst_input+filenames[3])
exampleImg = cv2.cvtColor(exampleImg, cv2.COLOR_BGR2RGB)
plt.imshow(exampleImg)
print('test3.jpg')
test3.jpg

Undistort image function

In [14]:
#  We use the camera distortion matrix mtx and dist to undistort the raw lane pictures
def undistort(img):
    undist = cv2.undistort(img, mtx, dist, None, mtx)
    return undist
In [15]:
exampleImg_undistort = undistort(exampleImg)

# Visualize undistortion
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(20,10))
f.subplots_adjust(hspace = .2, wspace=.05)
ax1.imshow(exampleImg)
ax1.set_title('Original Image test3', fontsize=30)
# exampleImg_undistort = cv2.cvtColor(exampleImg_undistort, cv2.COLOR_BGR2RGB)
ax2.imshow(exampleImg_undistort)
ax2.set_title('Undistorted Image test3', fontsize=30)
exampleImg_undistort = cv2.cvtColor(exampleImg_undistort, cv2.COLOR_RGB2BGR)
cv2.imwrite(path+output+'undistorted_image.png', exampleImg_undistort)
print('The comparison between the original and undistorted image test3') 
The comparison between the original and undistorted image test3

3 Apply the perspective transform to get the bird's eye view of the picture

In [16]:
def unwarp(img, src, dst):
    h,w = img.shape[:2]
    M = cv2.getPerspectiveTransform(src, dst)
    Minv = cv2.getPerspectiveTransform(dst, src)
    # use cv2.warpPerspective() to warp your image to a top-down view
    warped = cv2.warpPerspective(img, M, (w,h), flags=cv2.INTER_LINEAR)
    return warped, M, Minv
In [17]:
h,w = exampleImg_undistort.shape[:2]

# define source and destination points for transform
src = np.float32([(575,464),
                  (707,464), 
                  (258,682), 
                  (1049,682)])
dst = np.float32([(450,0),
                  (w-450,0),
                  (450,h),
                  (w-450,h)])
# exampleImg_undistort = cv2.cvtColor(exampleImg_undistort, cv2.COLOR_RGB2BGR)

# Visualize unwarp

f, (ax1, ax2) = plt.subplots(1, 2, figsize=(20,10))
f.subplots_adjust(hspace = .2, wspace=.05)
exampleImg_undistort = cv2.cvtColor(exampleImg_undistort, cv2.COLOR_BGR2RGB)
ax1.imshow(exampleImg_undistort)
x = [src[0][0],src[2][0],src[3][0],src[1][0],src[0][0]]
y = [src[0][1],src[2][1],src[3][1],src[1][1],src[0][1]]
ax1.plot(x, y, color='#33cc99', alpha=0.4, linewidth=3, solid_capstyle='round', zorder=2)
ax1.set_ylim([h,0])
ax1.set_xlim([0,w])
ax1.set_title('Undistorted Image', fontsize=30)
exampleImg_undistort = cv2.cvtColor(exampleImg_undistort, cv2.COLOR_RGB2BGR)
exampleImg_unwarp, M, Minv = unwarp(exampleImg_undistort, src, dst)
exampleImg_unwarp = cv2.cvtColor(exampleImg_unwarp, cv2.COLOR_BGR2RGB)
ax2.imshow(exampleImg_unwarp)
ax2.set_title('Unwarped Image', fontsize=30)
print('Comparison between the undistorted and the unwarped image ')
Comparison between the undistorted and the unwarped image 
In [18]:
h,w
Out[18]:
(720, 1280)

4 Use color transforms, gradients, etc., to create a thresholded binary image.

Consider firstly the color transform to create thresholded binary images.

In [19]:
# Visualize multiple color space channels
exampleImg_unwarp_R = exampleImg_unwarp[:,:,0]
exampleImg_unwarp_G = exampleImg_unwarp[:,:,1]
exampleImg_unwarp_B = exampleImg_unwarp[:,:,2]
exampleImg_unwarp_HSV = cv2.cvtColor(exampleImg_unwarp, cv2.COLOR_RGB2HSV)
exampleImg_unwarp_H = exampleImg_unwarp_HSV[:,:,0]
exampleImg_unwarp_S = exampleImg_unwarp_HSV[:,:,1]
exampleImg_unwarp_V = exampleImg_unwarp_HSV[:,:,2]
exampleImg_unwarp_LAB = cv2.cvtColor(exampleImg_unwarp, cv2.COLOR_RGB2Lab)
exampleImg_unwarp_L = exampleImg_unwarp_LAB[:,:,0]
exampleImg_unwarp_A = exampleImg_unwarp_LAB[:,:,1]
exampleImg_unwarp_B2 = exampleImg_unwarp_LAB[:,:,2]
fig, axs = plt.subplots(3,3, figsize=(16, 12))
fig.subplots_adjust(hspace = .2, wspace=.001)
axs = axs.ravel()
axs[0].imshow(exampleImg_unwarp_R, cmap='gray')
cv2.imwrite(path+output_color+'unwarp_R.jpg', exampleImg_unwarp_R)
axs[0].set_title('RGB R-channel', fontsize=30)
axs[1].imshow(exampleImg_unwarp_G, cmap='gray')
cv2.imwrite(path+output_color+'unwarp_G.jpg', exampleImg_unwarp_G)
axs[1].set_title('RGB G-Channel', fontsize=30)
axs[2].imshow(exampleImg_unwarp_B, cmap='gray')
cv2.imwrite(path+output_color+'unwarp_B.jpg', exampleImg_unwarp_B)
axs[2].set_title('RGB B-channel', fontsize=30)
axs[3].imshow(exampleImg_unwarp_H, cmap='gray')
cv2.imwrite(path+output_color+'unwarp_H.jpg', exampleImg_unwarp_H)
axs[3].set_title('HSV H-Channel', fontsize=30)
axs[4].imshow(exampleImg_unwarp_S, cmap='gray')
cv2.imwrite(path+output_color+'unwarp_S.jpg', exampleImg_unwarp_S)
axs[4].set_title('HSV S-channel', fontsize=30)
axs[5].imshow(exampleImg_unwarp_V, cmap='gray')
cv2.imwrite(path+output_color+'unwarp_V.jpg', exampleImg_unwarp_V)
axs[5].set_title('HSV V-Channel', fontsize=30)
axs[6].imshow(exampleImg_unwarp_L, cmap='gray')
cv2.imwrite(path+output_color+'unwarp_L.jpg', exampleImg_unwarp_L)
axs[6].set_title('LAB L-channel', fontsize=30)
axs[7].imshow(exampleImg_unwarp_A, cmap='gray')
cv2.imwrite(path+output_color+'unwarp_A.jpg', exampleImg_unwarp_A)
axs[7].set_title('LAB A-Channel', fontsize=30)
axs[8].imshow(exampleImg_unwarp_B2, cmap='gray')
cv2.imwrite(path+output_color+'unwarp_B2.jpg', exampleImg_unwarp_B2)
axs[8].set_title('LAB B-Channel', fontsize=30)
print('Comparison between the binary pictures under different color transform method')
Comparison between the binary pictures under different color transform method

Sobel Absolute Threshold

In [20]:
# Sobel Absolute Threshold
def abs_sobel_thresh(img, orient = 'x', thresh_min = 25, thresh_max = 255):
    gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
    sobel =  cv2.Sobel(gray, cv2.CV_64F, orient=='x', orient == 'y')
    abs_sobel = np.absolute(sobel)
    scale_factor = np.max(abs_sobel) / 255
    scaled_sobel = (abs_sobel / scale_factor).astype(np.uint8)
    sxbinary = np.zeros_like(scaled_sobel)
    sxbinary[(scaled_sobel >= thresh_min) & (scaled_sobel <= thresh_max)] = 1
    binary_output = sxbinary
    return binary_output

exampleImg_unwarp = cv2.cvtColor(exampleImg_unwarp, cv2.COLOR_RGB2BGR)

exampleImg_sobelAbs = abs_sobel_thresh(exampleImg_unwarp, 'x', 25, 255)
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(20,10))
f.subplots_adjust(hspace = .2, wspace=.05)
exampleImg_unwarp = cv2.cvtColor(exampleImg_unwarp, cv2.COLOR_BGR2RGB)
ax1.imshow(exampleImg_unwarp)
ax1.set_title('Unwarped Image', fontsize=30)
# exampleImg_sobelAbs = cv2.cvtColor(exampleImg_sobelAbs, cv2.COLOR_BGR2RGB)
ax2.imshow(exampleImg_sobelAbs, cmap='gray')
ax2.set_title('Sobel Absolute', fontsize=30)
print('Comparison between unwarped and the sobel absolute in x direction')
Comparison between unwarped and the sobel absolute in x direction

Sobel Magnitude Threshold

In [21]:
# Define a function that applies Sobel x and y to calculate the magnitude
# of the gradient and applies a threshold
def mag_thresh(img, sobel_kernel = 3, thresh = (105, 255)):
    # Convert to grayscale
    gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
    # Take both Sobel x and y gradients
    sobelx = cv2.Sobel(gray, cv2.CV_64F, 1, 0, ksize=sobel_kernel)
    sobely = cv2.Sobel(gray, cv2.CV_64F, 0, 1, ksize=sobel_kernel)
    # Calculate the gradient magnitude
    gradmag = np.sqrt(sobelx**2 + sobely**2)
    # Rescale to 8 bit
    scale_factor = np.max(gradmag)/255 
    gradmag = (gradmag/scale_factor).astype(np.uint8) 
    # Create a binary image of ones where threshold is met, zeros otherwise
    binary_output = np.zeros_like(gradmag)
    binary_output[(gradmag >= thresh[0]) & (gradmag <= thresh[1])] = 1
    return binary_output

exampleImg_unwarp = cv2.cvtColor(exampleImg_unwarp, cv2.COLOR_RGB2BGR)

exampleImg_sobelMag = mag_thresh(exampleImg_unwarp, sobel_kernel = 5, thresh = (30, 255))
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(20,10))
f.subplots_adjust(hspace = .2, wspace=.05)

exampleImg_unwarp = cv2.cvtColor(exampleImg_unwarp, cv2.COLOR_BGR2RGB)

ax1.imshow(exampleImg_unwarp)
ax1.set_title('Unwarped Image', fontsize=30)
ax2.imshow(exampleImg_sobelMag, cmap='gray')
ax2.set_title('Sobel Magnitude', fontsize=30)
print('Comparison between unwarped and the sobel absolute in magnitude')
Comparison between unwarped and the sobel absolute in magnitude

Sobel Direction Method

In [22]:
# Define a function that applies Sobel x and y, 
# then computes the direction of the gradient
# and applies a threshold.
def dir_thresh(img, sobel_kernel=7, thresh=(0, 0.09)):    
    # Apply the following steps to img
    # 1) Convert to grayscale
    gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
    # 2) Take the gradient in x and y separately
    sobelx = cv2.Sobel(gray, cv2.CV_64F, 1, 0, ksize=sobel_kernel)
    sobely = cv2.Sobel(gray, cv2.CV_64F, 0, 1, ksize=sobel_kernel)
    # 3) Take the absolute value of the x and y gradients
    abs_sobelx = np.absolute(sobelx)
    abs_sobely = np.absolute(sobely)
    # 4) Use np.arctan2(abs_sobely, abs_sobelx) to calculate the direction of the gradient 
    grad_dir = np.arctan2(abs_sobely, abs_sobelx)
    # 5) Create a binary mask where direction thresholds are met
    binary_output =  np.zeros_like(grad_dir)
    binary_output[(grad_dir >= thresh[0]) & (grad_dir <= thresh[1])] = 1
    # 6) Return this mask as your binary_output image
    return binary_output

exampleImg_unwarp = cv2.cvtColor(exampleImg_unwarp, cv2.COLOR_RGB2BGR)

exampleImg_sobelDir = dir_thresh(exampleImg_unwarp, sobel_kernel = 5, thresh = (np.pi/90, np.pi/30))
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(20,10))
f.subplots_adjust(hspace = .2, wspace=.05)
exampleImg_unwarp = cv2.cvtColor(exampleImg_unwarp, cv2.COLOR_BGR2RGB)

ax1.imshow(exampleImg_unwarp)
ax1.set_title('Unwarped Image', fontsize=30)
ax2.imshow(exampleImg_sobelDir, cmap='gray')
ax2.set_title('Sobel Direction', fontsize=30)
print('The comparison between the unwarped image and the image under Sobel direction transform')
The comparison between the unwarped image and the image under Sobel direction transform

HLS-S-Channel

In [23]:
# Define a function that thresholds the S-channel of HLS
# Use exclusive lower bound (>) and inclusive upper (<=)
def hls_sthresh(img, thresh = (100, 255)):
    hls = cv2.cvtColor(img, cv2.COLOR_RGB2HLS)
    S = hls[:, :, 2]
    binary_output = np.zeros_like(S)
    binary_output[(S > thresh[0]) & (S <= thresh[1])] = 1
    return binary_output

exampleImg_unwarp = cv2.cvtColor(exampleImg_unwarp, cv2.COLOR_RGB2BGR)

exampleImg_SThresh = hls_sthresh(exampleImg_unwarp, thresh = (140, 255))
# Visualize hls S-channel threshold
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(20,10))
f.subplots_adjust(hspace = .2, wspace=.05)
exampleImg_unwarp = cv2.cvtColor(exampleImg_unwarp, cv2.COLOR_BGR2RGB)

ax1.imshow(exampleImg_unwarp)
ax1.set_title('Unwarped Image', fontsize=30)
ax2.imshow(exampleImg_SThresh, cmap='gray')
ax2.set_title('HLS S-Channel', fontsize=30)
print('Comparison between the unwarped and the HLS-S channel threshold')
Comparison between the unwarped and the HLS-S channel threshold

HLS-L-Channel

In [24]:
# Define a function that thresholds the L-channel of HLS
# Use exclusive lower bound (>) and inclusive upper (<=)
def hls_lthresh(img, thresh = (220, 255)):
    hls = cv2.cvtColor(img, cv2.COLOR_RGB2HLS)
    L = hls[:, :, 1]
    binary_output = np.zeros_like(L)
    binary_output[(L > thresh[0]) & (L <= thresh[1])] = 1
    return binary_output

exampleImg_unwarp = cv2.cvtColor(exampleImg_unwarp, cv2.COLOR_RGB2BGR)

exampleImg_LThresh = hls_lthresh(exampleImg_unwarp, thresh = (220, 255))
# Visualize hls l-channel threshold
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(20,10))
f.subplots_adjust(hspace = .2, wspace=.05)
exampleImg_unwarp = cv2.cvtColor(exampleImg_unwarp, cv2.COLOR_BGR2RGB)
ax1.imshow(exampleImg_unwarp)
ax1.set_title('Unwarped Image', fontsize=30)
ax2.imshow(exampleImg_LThresh, cmap='gray')
ax2.set_title('HLS L-Channel', fontsize=30)
print('Comparison between the unwarped and the HLS-L channel threshold')
Comparison between the unwarped and the HLS-L channel threshold

LAB Colorspace B-Channel

In [25]:
# Define a function that thresholds the LAB-color space
# Use exclusive lower bound (>) and inclusive upper (<=)
def lab_bthresh(img, thresh = (190, 255)):
    # 1) Convert to LAB color space
    lab = cv2.cvtColor(img, cv2.COLOR_RGB2Lab)
    lab_b = lab[:,:,2]
    # don't normalize if there are no yellows in the image
    if np.max(lab_b) > 175:
        lab_b = lab_b*(255/np.max(lab_b))
    # 2) Apply a threshold to the L channel
    binary_output = np.zeros_like(lab_b)
    binary_output[((lab_b > thresh[0]) & (lab_b <= thresh[1]))] = 1
    # 3) Return a binary image of threshold result
    return binary_output

exampleImg_unwarp = cv2.cvtColor(exampleImg_unwarp, cv2.COLOR_RGB2BGR)

exampleImg_LBThresh = lab_bthresh(exampleImg_unwarp, thresh = (190, 255))
# Visualize hls l-channel threshold
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(20,10))
f.subplots_adjust(hspace = .2, wspace=.05)
exampleImg_unwarp = cv2.cvtColor(exampleImg_unwarp, cv2.COLOR_BGR2RGB)
ax1.imshow(exampleImg_unwarp)
ax1.set_title('Unwarped Image', fontsize=30)
ax2.imshow(exampleImg_LBThresh, cmap='gray')
ax2.set_title('LAB B-Channel', fontsize=30)
print('Comparison between the unwarped and the LAB-B channel threshold')
Comparison between the unwarped and the LAB-B channel threshold

It might be interesting to combine the magnitude and the direction method to do some research.

HLS-L-Channel +LAB-B-Channel

In [26]:
def combined_thresh(img, thresh1 = (220, 255), thresh2 = (190, 255)):
    img_LThresh = hls_lthresh(img, thresh = thresh1)
#     exampleImg_SThresh = hls_sthresh(exampleImg_unwarp, thresh = thresh1)
    img_BThresh = lab_bthresh(img, thresh = thresh2)
    combined = np.zeros_like(exampleImg_LThresh)
    combined[(( img_LThresh == 1) | (img_BThresh == 1))] = 1
    return combined
#   & (exampleImg_sobelDir2 == 1) 

exampleImg_unwarp = cv2.cvtColor(exampleImg_unwarp, cv2.COLOR_RGB2BGR)

exampleImg_combined = combined_thresh(exampleImg_unwarp)
# cv2.imwrite(path+output_combined+'combined.jpg', exampleImg_combined)
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(20,10))
f.subplots_adjust(hspace = .2, wspace=.05)
exampleImg_unwarp = cv2.cvtColor(exampleImg_unwarp, cv2.COLOR_BGR2RGB)
ax1.imshow(exampleImg_unwarp)
ax1.set_title('Unwarped Image', fontsize=30)
ax2.imshow(exampleImg_combined, cmap='gray')
ax2.set_title('HLS-L-Channel + Lab-B-Channel', fontsize=30)
print('The comparison between unwarped image and combined binary image')
The comparison between unwarped image and combined binary image

Define Image Processing Pipeline

In [27]:
# # Define the complete image processing pipeline, reads raw image and returns binary image with lane lines identified
# # (hopefully)
# def pipeline(img):
#     # Undistort
#     img_undistort = undistort(img)
    
#     # Perspective Transform
#     img_unwarp, M, Minv = unwarp(img_undistort, src, dst)

#     # Sobel Absolute (using default parameters)
#     #img_sobelAbs = abs_sobel_thresh(img_unwarp)

#     # Sobel Magnitude (using default parameters)
#     #img_sobelMag = mag_thresh(img_unwarp)
    
#     # Sobel Direction (using default parameters)
#     #img_sobelDir = dir_thresh(img_unwarp)
    
#     # HLS S-channel Threshold (using default parameters)
#     #img_SThresh = hls_sthresh(img_unwarp)

#     # HLS L-channel Threshold (using default parameters)
#     img_LThresh = hls_lthresh(img_unwarp)

#     # Lab B-channel Threshold (using default parameters)
#     img_BThresh = lab_bthresh(img_unwarp)
    
#     # Combine HLS and Lab B channel thresholds
#     combined = np.zeros_like(img_BThresh)
#     combined[(img_LThresh == 1) | (img_BThresh == 1)] = 1
#     return combined, Minv
In [28]:
# To conclude the result of the previous 4 steps, 
# we can define image processing pipeline to read a batch of pictures.

def pipeline(img):
    # unditort the image using the 
    img_undistort = undistort(img)
    # Perspective Transform in order to get a so called 'bird eye view'
    img_unwarp, M, Minv = unwarp(img_undistort, src, dst)
    # Consider the HLS-Smethod
    combined= combined_thresh(img_unwarp)
    return combined, Minv

# Make a list of example images
images = glob.glob('./test_images/*.jpg')
                                          
# Set up plot
fig, axs = plt.subplots(len(images),2, figsize=(10, 20))
fig.subplots_adjust(hspace = .2, wspace=.001)
axs = axs.ravel()
                  
i = 0
for image in images:
    img = cv2.imread(image)
    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
    img_bin, Minv = pipeline(img)
    axs[i].imshow(img)
    axs[i].axis('off')
    i += 1
    axs[i].imshow(img_bin, cmap='gray')
    axs[i].axis('off')
    i += 1

    
print('The Pipeline on all test images')
The Pipeline on all test images

5 Detect lane pixels and fit to find the lane boundary

In [29]:
# Define method to fit polynomial to binary image with lines extracted, using sliding window
def sliding_window_polyfit(img):
    # Take a histogram of the bottom half of the image
    histogram = np.sum(img[img.shape[0]//2:,:], axis=0)
    # Find the peak of the left and right halves of the histogram
    # These will be the starting point for the left and right lines
    midpoint = np.int(histogram.shape[0]//2)
    quarter_point = np.int(midpoint//2)
    # Previously the left/right base was the max of the left/right half of the histogram
    # this changes it so that only a quarter of the histogram (directly to the left/right) is considered
    leftx_base = np.argmax(histogram[quarter_point:midpoint]) + quarter_point
    rightx_base = np.argmax(histogram[midpoint:(midpoint+quarter_point)]) + midpoint
    
    #print('base pts:', leftx_base, rightx_base)

    # Choose the number of sliding windows
    nwindows = 10
    # Set height of windows
    window_height = np.int(img.shape[0]/nwindows)
    # Identify the x and y positions of all nonzero pixels in the image
    nonzero = img.nonzero()
    nonzeroy = np.array(nonzero[0])
    nonzerox = np.array(nonzero[1])
    # Current positions to be updated for each window
    leftx_current = leftx_base
    rightx_current = rightx_base
    # Set the width of the windows +/- margin
    margin = 80
    # Set minimum number of pixels found to recenter window
    minpix = 40
    # Create empty lists to receive left and right lane pixel indices
    left_lane_inds = []
    right_lane_inds = []
    # Rectangle data for visualization
    rectangle_data = []

    # Step through the windows one by one
    for window in range(nwindows):
        # Identify window boundaries in x and y (and right and left)
        win_y_low = img.shape[0] - (window+1)*window_height
        win_y_high = img.shape[0] - window*window_height
        win_xleft_low = leftx_current - margin
        win_xleft_high = leftx_current + margin
        win_xright_low = rightx_current - margin
        win_xright_high = rightx_current + margin
        rectangle_data.append((win_y_low, win_y_high, win_xleft_low, win_xleft_high, win_xright_low, win_xright_high))
        # Identify the nonzero pixels in x and y within the window
        good_left_inds = ((nonzeroy >= win_y_low) & (nonzeroy < win_y_high) & (nonzerox >= win_xleft_low) & (nonzerox < win_xleft_high)).nonzero()[0]
        good_right_inds = ((nonzeroy >= win_y_low) & (nonzeroy < win_y_high) & (nonzerox >= win_xright_low) & (nonzerox < win_xright_high)).nonzero()[0]
        # Append these indices to the lists
        left_lane_inds.append(good_left_inds)
        right_lane_inds.append(good_right_inds)
        # If you found > minpix pixels, recenter next window on their mean position
        if len(good_left_inds) > minpix:
            leftx_current = np.int(np.mean(nonzerox[good_left_inds]))
        if len(good_right_inds) > minpix:        
            rightx_current = np.int(np.mean(nonzerox[good_right_inds]))

    # Concatenate the arrays of indices
    left_lane_inds = np.concatenate(left_lane_inds)
    right_lane_inds = np.concatenate(right_lane_inds)

    # Extract left and right line pixel positions
    leftx = nonzerox[left_lane_inds]
    lefty = nonzeroy[left_lane_inds] 
    rightx = nonzerox[right_lane_inds]
    righty = nonzeroy[right_lane_inds] 

    left_fit, right_fit = (None, None)
    # Fit a second order polynomial to each
    if len(leftx) != 0:
        left_fit = np.polyfit(lefty, leftx, 2)
    if len(rightx) != 0:
        right_fit = np.polyfit(righty, rightx, 2)
    
    visualization_data = (rectangle_data, histogram)
    
    return left_fit, right_fit, left_lane_inds, right_lane_inds, visualization_data
print('Define method to fit polynomial to binary image with lines extracted, using sliding window')
Define method to fit polynomial to binary image with lines extracted, using sliding window

Visualize the sliding window polifit on example image

In [30]:
# visualize the result on example image
exampleImg = cv2.imread(path + output + filenames[3])
exampleImg = cv2.cvtColor(exampleImg, cv2.COLOR_BGR2RGB)
exampleImg_bin, Minv = pipeline(exampleImg)
    
left_fit, right_fit, left_lane_inds, right_lane_inds, visualization_data = sliding_window_polyfit(exampleImg_bin)

h = exampleImg.shape[0]
left_fit_x_int = left_fit[0]*h**2 + left_fit[1]*h + left_fit[2]
right_fit_x_int = right_fit[0]*h**2 + right_fit[1]*h + right_fit[2]
#print('fit x-intercepts:', left_fit_x_int, right_fit_x_int)

rectangles = visualization_data[0]
histogram = visualization_data[1]

# Create an output image to draw on and  visualize the result
out_img = np.uint8(np.dstack((exampleImg_bin, exampleImg_bin, exampleImg_bin))*255)
# Generate x and y values for plotting
ploty = np.linspace(0, exampleImg_bin.shape[0]-1, exampleImg_bin.shape[0] )
left_fitx = left_fit[0]*ploty**2 + left_fit[1]*ploty + left_fit[2]
right_fitx = right_fit[0]*ploty**2 + right_fit[1]*ploty + right_fit[2]
for rect in rectangles:
# Draw the windows on the visualization image
    cv2.rectangle(out_img,(rect[2],rect[0]),(rect[3],rect[1]),(0,255,0), 2) 
    cv2.rectangle(out_img,(rect[4],rect[0]),(rect[5],rect[1]),(0,255,0), 2) 
# Identify the x and y positions of all nonzero pixels in the image
nonzero = exampleImg_bin.nonzero()
nonzeroy = np.array(nonzero[0])
nonzerox = np.array(nonzero[1])
out_img[nonzeroy[left_lane_inds], nonzerox[left_lane_inds]] = [255, 0, 0]
out_img[nonzeroy[right_lane_inds], nonzerox[right_lane_inds]] = [100, 200, 255]
plt.imshow(out_img)
plt.plot(left_fitx, ploty, color='yellow')
plt.plot(right_fitx, ploty, color='yellow')
plt.xlim(0, 1280)
plt.ylim(720, 0)
Out[30]:
(720, 0)
In [31]:
# Print histogram from sliding window polyfit for example image
plt.plot(histogram)
plt.xlim(0, 1280)
print('...')
...
In [32]:
# Define method to fit polynomial to binary image based upon a previous fit;
# However, this should assume the fit will not change too much from one video frame to the next.

# Define method to fit polynomial to binary image based upon a previous fit (chronologically speaking);
# this assumes that the fit will not change significantly from one video frame to the next
def polyfit_using_prev_fit(binary_warped, left_fit_prev, right_fit_prev):
    nonzero = binary_warped.nonzero()
    nonzeroy = np.array(nonzero[0])
    nonzerox = np.array(nonzero[1])
    margin = 80
    left_lane_inds = ((nonzerox > (left_fit_prev[0]*(nonzeroy**2) + left_fit_prev[1]*nonzeroy + left_fit_prev[2] - margin)) & 
                      (nonzerox < (left_fit_prev[0]*(nonzeroy**2) + left_fit_prev[1]*nonzeroy + left_fit_prev[2] + margin))) 
    right_lane_inds = ((nonzerox > (right_fit_prev[0]*(nonzeroy**2) + right_fit_prev[1]*nonzeroy + right_fit_prev[2] - margin)) & 
                       (nonzerox < (right_fit_prev[0]*(nonzeroy**2) + right_fit_prev[1]*nonzeroy + right_fit_prev[2] + margin)))  

    # Again, extract left and right line pixel positions
    leftx = nonzerox[left_lane_inds]
    lefty = nonzeroy[left_lane_inds] 
    rightx = nonzerox[right_lane_inds]
    righty = nonzeroy[right_lane_inds]
    
    left_fit_new, right_fit_new = (None, None)
    if len(leftx) != 0:
        # Fit a second order polynomial to each
        left_fit_new = np.polyfit(lefty, leftx, 2)
    if len(rightx) != 0:
        right_fit_new = np.polyfit(righty, rightx, 2)
    return left_fit_new, right_fit_new, left_lane_inds, right_lane_inds

# visualize the result on example image
exampleImg2 = cv2.imread(path+tst_input + filenames[3])
exampleImg2 = cv2.cvtColor(exampleImg2, cv2.COLOR_BGR2RGB)
exampleImg2_bin, Minv = pipeline(exampleImg2)   
margin = 80

left_fit2, right_fit2, left_lane_inds2, right_lane_inds2 = polyfit_using_prev_fit(exampleImg2_bin, left_fit, right_fit)

# Generate x and y values for plotting
ploty = np.linspace(0, exampleImg2_bin.shape[0]-1, exampleImg2_bin.shape[0] )
left_fitx = left_fit[0]*ploty**2 + left_fit[1]*ploty + left_fit[2]
right_fitx = right_fit[0]*ploty**2 + right_fit[1]*ploty + right_fit[2]
left_fitx2 = left_fit2[0]*ploty**2 + left_fit2[1]*ploty + left_fit2[2]
right_fitx2 = right_fit2[0]*ploty**2 + right_fit2[1]*ploty + right_fit2[2]

# Create an image to draw on and an image to show the selection window
out_img = np.uint8(np.dstack((exampleImg2_bin, exampleImg2_bin, exampleImg2_bin))*255)
window_img = np.zeros_like(out_img)

# Color in left and right line pixels
nonzero = exampleImg2_bin.nonzero()
nonzeroy = np.array(nonzero[0])
nonzerox = np.array(nonzero[1])
out_img[nonzeroy[left_lane_inds2], nonzerox[left_lane_inds2]] = [255, 0, 0]
out_img[nonzeroy[right_lane_inds2], nonzerox[right_lane_inds2]] = [0, 0, 255]

# Generate a polygon to illustrate the search window area (OLD FIT)
# And recast the x and y points into usable format for cv2.fillPoly()
left_line_window1 = np.array([np.transpose(np.vstack([left_fitx-margin, ploty]))])
left_line_window2 = np.array([np.flipud(np.transpose(np.vstack([left_fitx+margin, ploty])))])
left_line_pts = np.hstack((left_line_window1, left_line_window2))
right_line_window1 = np.array([np.transpose(np.vstack([right_fitx-margin, ploty]))])
right_line_window2 = np.array([np.flipud(np.transpose(np.vstack([right_fitx+margin, ploty])))])
right_line_pts = np.hstack((right_line_window1, right_line_window2))

# Draw the lane onto the warped blank image
cv2.fillPoly(window_img, np.int_([left_line_pts]), (0,255, 0))
cv2.fillPoly(window_img, np.int_([right_line_pts]), (0,255, 0))
result = cv2.addWeighted(out_img, 1, window_img, 0.3, 0)
plt.imshow(result)
plt.plot(left_fitx2, ploty, color='yellow')
plt.plot(right_fitx2, ploty, color='yellow')
plt.xlim(0, 1280)
plt.ylim(720, 0)

result = cv2.cvtColor(result, cv2.COLOR_RGB2BGR)
cv2.imwrite(path+output+'polynomial_fitted_lanes.png', result)
Out[32]:
True

6 Determine the curvature of the lane and vehicle position with respect to center.

In [33]:
# Method to determine radius of curvature and distance from lane center 
# based on binary image, polynomial fit, and L and R lane pixel indices
def calc_curv_rad_and_center_dist(bin_img, l_fit, r_fit, l_lane_inds, r_lane_inds):
    # Define conversions in x and y from pixels space to meters
    ym_per_pix = 3.048/100 # meters per pixel in y dimension, lane line is 10 ft = 3.048 meters
    xm_per_pix = 3.7/378 # meters per pixel in x dimension, lane width is 12 ft = 3.7 meters
    left_curverad, right_curverad, center_dist = (0, 0, 0)
    # Define y-value where we want radius of curvature
    # I'll choose the maximum y-value, corresponding to the bottom of the image
    h = bin_img.shape[0]
    ploty = np.linspace(0, h-1, h)
    y_eval = np.max(ploty)
  
    # Identify the x and y positions of all nonzero pixels in the image
    nonzero = bin_img.nonzero()
    nonzeroy = np.array(nonzero[0])
    nonzerox = np.array(nonzero[1])
    # Again, extract left and right line pixel positions
    leftx = nonzerox[l_lane_inds]
    lefty = nonzeroy[l_lane_inds] 
    rightx = nonzerox[r_lane_inds]
    righty = nonzeroy[r_lane_inds]
    
    if len(leftx) != 0 and len(rightx) != 0:
        # Fit new polynomials to x,y in world space
        left_fit_cr = np.polyfit(lefty*ym_per_pix, leftx*xm_per_pix, 2)
        right_fit_cr = np.polyfit(righty*ym_per_pix, rightx*xm_per_pix, 2)
        # Calculate the new radii of curvature
        left_curverad = ((1 + (2*left_fit_cr[0]*y_eval*ym_per_pix + left_fit_cr[1])**2)**1.5) / np.absolute(2*left_fit_cr[0])
        right_curverad = ((1 + (2*right_fit_cr[0]*y_eval*ym_per_pix + right_fit_cr[1])**2)**1.5) / np.absolute(2*right_fit_cr[0])
        # Now our radius of curvature is in meters
    
    # Distance from center is image x midpoint - mean of l_fit and r_fit intercepts 
    if r_fit is not None and l_fit is not None:
        car_position = bin_img.shape[1]/2
        l_fit_x_int = l_fit[0]*h**2 + l_fit[1]*h + l_fit[2]
        r_fit_x_int = r_fit[0]*h**2 + r_fit[1]*h + r_fit[2]
        lane_center_position = (r_fit_x_int + l_fit_x_int) /2
        center_dist = (car_position - lane_center_position) * xm_per_pix
    return left_curverad, right_curverad, center_dist


rad_l, rad_r, d_center = calc_curv_rad_and_center_dist(exampleImg_bin, left_fit, right_fit, left_lane_inds, right_lane_inds)

print('Radius of curvature for example:', rad_l, 'm,', rad_r, 'm')
print('Distance from lane center for example:', d_center, 'm')        
Radius of curvature for example: 448.03458377285017 m, 1114.7045990370355 m
Distance from lane center for example: -0.38688616378976093 m

7 Warp the detected lane boundaries back onto the original image.

In [34]:
def draw_lane(original_img, binary_img, l_fit, r_fit, Minv):
    new_img = np.copy(original_img)
    if l_fit is None or r_fit is None:
        return original_img
    # Create an image to draw the lines on
    warp_zero = np.zeros_like(binary_img).astype(np.uint8)
    color_warp = np.dstack((warp_zero, warp_zero, warp_zero))
    
    h,w = binary_img.shape
    ploty = np.linspace(0, h-1, num=h)# to cover same y-range as image
    left_fitx = l_fit[0]*ploty**2 + l_fit[1]*ploty + l_fit[2]
    right_fitx = r_fit[0]*ploty**2 + r_fit[1]*ploty + r_fit[2]

    # Recast the x and y points into usable format for cv2.fillPoly()
    pts_left = np.array([np.transpose(np.vstack([left_fitx, ploty]))])
    pts_right = np.array([np.flipud(np.transpose(np.vstack([right_fitx, ploty])))])
    pts = np.hstack((pts_left, pts_right))

    # Draw the lane onto the warped blank image
    cv2.fillPoly(color_warp, np.int_([pts]), (0,255, 0))
    cv2.polylines(color_warp, np.int32([pts_left]), isClosed=False, color=(255,0,255), thickness=15)
    cv2.polylines(color_warp, np.int32([pts_right]), isClosed=False, color=(0,255,255), thickness=15)

    # Warp the blank back to original image space using inverse perspective matrix (Minv)
    newwarp = cv2.warpPerspective(color_warp, Minv, (w, h)) 
    # Combine the result with the original image
    result = cv2.addWeighted(new_img, 1, newwarp, 0.5, 0)
    return result

exampleImg_out1 = draw_lane(exampleImg, exampleImg_bin, left_fit, right_fit, Minv)
plt.imshow(exampleImg_out1)
Out[34]:
<matplotlib.image.AxesImage at 0xd2b55c2e8>
In [35]:
# Draw curvature radius and distance from center data onto the original image

def draw_data(original_img, curv_rad, center_dist):
    new_img = np.copy(original_img)
    h = new_img.shape[0]
    font = cv2.FONT_HERSHEY_DUPLEX
    text = 'Curve radius: ' + '{:04.2f}'.format(curv_rad) + 'm'
    cv2.putText(new_img, text, (40,70), font, 1.5, (200,255,155), 2, cv2.LINE_AA)
    direction = ''
    if center_dist > 0:
        direction = 'right'
    elif center_dist < 0:
        direction = 'left'
    abs_center_dist = abs(center_dist)
    text = '{:04.3f}'.format(abs_center_dist) + 'm ' + direction + ' of center'
    cv2.putText(new_img, text, (40,120), font, 1.5, (200,255,155), 2, cv2.LINE_AA)
    return new_img

exampleImg_out2 = draw_data(exampleImg_out1, (rad_l + rad_r)/2, d_center)
plt.imshow(exampleImg_out2)
print('The data curvature radius and distance from center data onto the original image')

exampleImg_out2 = cv2.cvtColor(exampleImg_out2, cv2.COLOR_RGB2BGR)
cv2.imwrite(path + output + 'draw_back_radius_distance.png',exampleImg_out2)
The data curvature radius and distance from center data onto the original image
Out[35]:
True
In [36]:
# Define a class to receive the characteristics of each line detection
class Line():
    def __init__(self):
        # was the line detected in the last iteration?
        self.detected = False  
        # x values of the last n fits of the line
        self.recent_xfitted = [] 
        #average x values of the fitted line over the last n iterations
        self.bestx = None     
        #polynomial coefficients averaged over the last n iterations
        self.best_fit = None  
        #polynomial coefficients for the most recent fit
        self.current_fit = []  
        #radius of curvature of the line in some units
        self.radius_of_curvature = None 
        #distance in meters of vehicle center from the line
        self.line_base_pos = None 
        #difference in fit coefficients between last and new fits
        self.diffs = np.array([0,0,0], dtype='float') 
        #number of detected pixels
        self.px_count = None
    def add_fit(self, fit, inds):
        # add a found fit to the line, up to n
        if fit is not None:
            if self.best_fit is not None:
                # if we have a best fit, see how this new fit compares
                self.diffs = abs(fit-self.best_fit)
            if (self.diffs[0] > 0.001 or \
               self.diffs[1] > 1.0 or \
               self.diffs[2] > 100.) and \
               len(self.current_fit) > 0:
                # bad fit! abort! abort! ... well, unless there are no fits in the current_fit queue, then we'll take it
                self.detected = False
            else:
                self.detected = True
                self.px_count = np.count_nonzero(inds)
                self.current_fit.append(fit)
                if len(self.current_fit) > 5:
                    # throw out old fits, keep newest n
                    self.current_fit = self.current_fit[len(self.current_fit)-5:]
                self.best_fit = np.average(self.current_fit, axis=0)
        # or remove one from the history, if not found
        else:
            self.detected = False
            if len(self.current_fit) > 0:
                # throw out oldest fit
                self.current_fit = self.current_fit[:len(self.current_fit)-1]
            if len(self.current_fit) > 0:
                # if there are still any fits in the queue, best_fit is their average
                self.best_fit = np.average(self.current_fit, axis=0)


        

Define complete image processing pipeline

In [37]:
def process_image(img):
    new_img = np.copy(img)
    img_bin, Minv = pipeline(new_img)
    
    # if both left and right lines were detected last frame, use polyfit_using_prev_fit, otherwise use sliding window
    if not l_line.detected or not r_line.detected:
        l_fit, r_fit, l_lane_inds, r_lane_inds, _ = sliding_window_polyfit(img_bin)
    else:
        l_fit, r_fit, l_lane_inds, r_lane_inds = polyfit_using_prev_fit(img_bin, l_line.best_fit, r_line.best_fit)
        
    # invalidate both fits if the difference in their x-intercepts isn't around 350 px (+/- 100 px)
    if l_fit is not None and r_fit is not None:
        # calculate x-intercept (bottom of image, x=image_height) for fits
        h = img.shape[0]
        l_fit_x_int = l_fit[0]*h**2 + l_fit[1]*h + l_fit[2]
        r_fit_x_int = r_fit[0]*h**2 + r_fit[1]*h + r_fit[2]
        x_int_diff = abs(r_fit_x_int-l_fit_x_int)
        if abs(350 - x_int_diff) > 100:
            l_fit = None
            r_fit = None
            
    l_line.add_fit(l_fit, l_lane_inds)
    r_line.add_fit(r_fit, r_lane_inds)
    
    # draw the current best fit if it exists
    if l_line.best_fit is not None and r_line.best_fit is not None:
        img_out1 = draw_lane(new_img, img_bin, l_line.best_fit, r_line.best_fit, Minv)
        rad_l, rad_r, d_center = calc_curv_rad_and_center_dist(img_bin, l_line.best_fit, r_line.best_fit, 
                                                               l_lane_inds, r_lane_inds)
        img_out = draw_data(img_out1, (rad_l+rad_r)/2, d_center)
    else:
        img_out = new_img
    
    diagnostic_output = False
    if diagnostic_output:
        # put together multi-view output
        diag_img = np.zeros((720,1280,3), dtype=np.uint8)
        
        # original output (top left)
        diag_img[0:360,0:640,:] = cv2.resize(img_out,(640,360))
        
        # binary overhead view (top right)
        img_bin = np.dstack((img_bin*255, img_bin*255, img_bin*255))
        resized_img_bin = cv2.resize(img_bin,(640,360))
        diag_img[0:360,640:1280, :] = resized_img_bin
        
        # overhead with all fits added (bottom right)
        img_bin_fit = np.copy(img_bin)
        for i, fit in enumerate(l_line.current_fit):
            img_bin_fit = plot_fit_onto_img(img_bin_fit, fit, (20*i+100,0,20*i+100))
        for i, fit in enumerate(r_line.current_fit):
            img_bin_fit = plot_fit_onto_img(img_bin_fit, fit, (0,20*i+100,20*i+100))
        img_bin_fit = plot_fit_onto_img(img_bin_fit, l_line.best_fit, (255,255,0))
        img_bin_fit = plot_fit_onto_img(img_bin_fit, r_line.best_fit, (255,255,0))
        diag_img[360:720,640:1280,:] = cv2.resize(img_bin_fit,(640,360))
        
        # diagnostic data (bottom left)
        color_ok = (200,255,155)
        color_bad = (255,155,155)
        font = cv2.FONT_HERSHEY_DUPLEX
        if l_fit is not None:
            text = 'This fit L: ' + ' {:0.6f}'.format(l_fit[0]) + \
                                    ' {:0.6f}'.format(l_fit[1]) + \
                                    ' {:0.6f}'.format(l_fit[2])
        else:
            text = 'This fit L: None'
        cv2.putText(diag_img, text, (40,380), font, .5, color_ok, 1, cv2.LINE_AA)
        if r_fit is not None:
            text = 'This fit R: ' + ' {:0.6f}'.format(r_fit[0]) + \
                                    ' {:0.6f}'.format(r_fit[1]) + \
                                    ' {:0.6f}'.format(r_fit[2])
        else:
            text = 'This fit R: None'
        cv2.putText(diag_img, text, (40,400), font, .5, color_ok, 1, cv2.LINE_AA)
        text = 'Best fit L: ' + ' {:0.6f}'.format(l_line.best_fit[0]) + \
                                ' {:0.6f}'.format(l_line.best_fit[1]) + \
                                ' {:0.6f}'.format(l_line.best_fit[2])
        cv2.putText(diag_img, text, (40,440), font, .5, color_ok, 1, cv2.LINE_AA)
        text = 'Best fit R: ' + ' {:0.6f}'.format(r_line.best_fit[0]) + \
                                ' {:0.6f}'.format(r_line.best_fit[1]) + \
                                ' {:0.6f}'.format(r_line.best_fit[2])
        cv2.putText(diag_img, text, (40,460), font, .5, color_ok, 1, cv2.LINE_AA)
        text = 'Diffs L: ' + ' {:0.6f}'.format(l_line.diffs[0]) + \
                             ' {:0.6f}'.format(l_line.diffs[1]) + \
                             ' {:0.6f}'.format(l_line.diffs[2])
        if l_line.diffs[0] > 0.001 or \
           l_line.diffs[1] > 1.0 or \
           l_line.diffs[2] > 100.:
            diffs_color = color_bad
        else:
            diffs_color = color_ok
        cv2.putText(diag_img, text, (40,500), font, .5, diffs_color, 1, cv2.LINE_AA)
        text = 'Diffs R: ' + ' {:0.6f}'.format(r_line.diffs[0]) + \
                             ' {:0.6f}'.format(r_line.diffs[1]) + \
                             ' {:0.6f}'.format(r_line.diffs[2])
        if r_line.diffs[0] > 0.001 or \
           r_line.diffs[1] > 1.0 or \
           r_line.diffs[2] > 100.:
            diffs_color = color_bad
        else:
            diffs_color = color_ok
        cv2.putText(diag_img, text, (40,520), font, .5, diffs_color, 1, cv2.LINE_AA)
        text = 'Good fit count L:' + str(len(l_line.current_fit))
        cv2.putText(diag_img, text, (40,560), font, .5, color_ok, 1, cv2.LINE_AA)
        text = 'Good fit count R:' + str(len(r_line.current_fit))
        cv2.putText(diag_img, text, (40,580), font, .5, color_ok, 1, cv2.LINE_AA)
        
        img_out = diag_img
    return img_out


def plot_fit_onto_img(img, fit, plot_color):
    if fit is None:
        return img
    new_img = np.copy(img)
    h = new_img.shape[0]
    ploty = np.linspace(0, h-1, h)
    plotx = fit[0]*ploty**2 + fit[1]*ploty + fit[2]
    pts = np.array([np.transpose(np.vstack([plotx, ploty]))])
    cv2.polylines(new_img, np.int32([pts]), isClosed=False, color=plot_color, thickness=8)
    return new_img

8 Output visual display of the lane boundaries and numerical estimation of lane curvature and vehicle position.

In [38]:
l_line = Line()
r_line = Line()
#my_clip.write_gif('test.gif', fps=12)
video_output1 = path + output_v + 'project_video_output.mp4'
video_input1 = VideoFileClip('project_video.mp4')#.subclip(22,26)
processed_video = video_input1.fl_image(process_image)
%time processed_video.write_videofile(video_output1, audio=False)
[MoviePy] >>>> Building video /Users/libo/Documents/Test/self_driving/project2_advanced_lane_finding/CarND-Advanced-Lane-Lines/output_videos/project_video_output.mp4
[MoviePy] Writing video /Users/libo/Documents/Test/self_driving/project2_advanced_lane_finding/CarND-Advanced-Lane-Lines/output_videos/project_video_output.mp4
100%|█████████▉| 1260/1261 [01:51<00:00, 10.92it/s]
[MoviePy] Done.
[MoviePy] >>>> Video ready: /Users/libo/Documents/Test/self_driving/project2_advanced_lane_finding/CarND-Advanced-Lane-Lines/output_videos/project_video_output.mp4 

CPU times: user 7min 55s, sys: 20.2 s, total: 8min 15s
Wall time: 1min 52s
In [39]:
HTML("""
<video width="960" height="540" controls>
  <source src="{0}">
</video>
""".format("output_videos/project_video_output.mp4"))
Out[39]:
In [40]:
l_line = Line()
r_line = Line()
#my_clip.write_gif('test.gif', fps=12)
video_output1 = path + output_v + 'challenge_video_output.mp4'
video_input1 = VideoFileClip('challenge_video.mp4')#.subclip(22,26)
processed_video = video_input1.fl_image(process_image)
%time processed_video.write_videofile(video_output1, audio=False)
[MoviePy] >>>> Building video /Users/libo/Documents/Test/self_driving/project2_advanced_lane_finding/CarND-Advanced-Lane-Lines/output_videos/challenge_video_output.mp4
[MoviePy] Writing video /Users/libo/Documents/Test/self_driving/project2_advanced_lane_finding/CarND-Advanced-Lane-Lines/output_videos/challenge_video_output.mp4
100%|██████████| 485/485 [00:39<00:00, 12.19it/s]
[MoviePy] Done.
[MoviePy] >>>> Video ready: /Users/libo/Documents/Test/self_driving/project2_advanced_lane_finding/CarND-Advanced-Lane-Lines/output_videos/challenge_video_output.mp4 

CPU times: user 2min 52s, sys: 6.39 s, total: 2min 58s
Wall time: 40.3 s
In [41]:
HTML("""
<video width="960" height="540" controls>
  <source src="{0}">
</video>
""".format("output_videos/challenge_video_output.mp4"))
Out[41]:
In [42]:
l_line = Line()
r_line = Line()
#my_clip.write_gif('test.gif', fps=12)
video_output1 = path + output_v + 'harder_challenge_video_output.mp4'
video_input1 = VideoFileClip('harder_challenge_video.mp4')#.subclip(22,26)
processed_video = video_input1.fl_image(process_image)
%time processed_video.write_videofile(video_output1, audio=False)
[MoviePy] >>>> Building video /Users/libo/Documents/Test/self_driving/project2_advanced_lane_finding/CarND-Advanced-Lane-Lines/output_videos/harder_challenge_video_output.mp4
[MoviePy] Writing video /Users/libo/Documents/Test/self_driving/project2_advanced_lane_finding/CarND-Advanced-Lane-Lines/output_videos/harder_challenge_video_output.mp4
100%|█████████▉| 1199/1200 [02:03<00:00,  8.95it/s]
[MoviePy] Done.
[MoviePy] >>>> Video ready: /Users/libo/Documents/Test/self_driving/project2_advanced_lane_finding/CarND-Advanced-Lane-Lines/output_videos/harder_challenge_video_output.mp4 

CPU times: user 8min 37s, sys: 20.6 s, total: 8min 57s
Wall time: 2min 4s
In [43]:
HTML("""
<video width="960" height="540" controls>
  <source src="{0}">
</video>
""".format("output_videos/harder_challenge_video_output.mp4"))
Out[43]:
In [ ]:
 
In [ ]:
 
In [44]:
load_file = open("wide_dist_pickle.p", 'rb+')
employee_dict = pickle.load(load_file)
employee_dict['dist']
Out[44]:
array([[-2.32949182e-01,  6.17242707e-02, -1.80423444e-05,
         3.39635746e-05, -7.54961807e-03]])
In [45]:
employee_dict.keys()
Out[45]:
dict_keys(['imgpoints', 'dist', 'objpoints', 'mtx'])
In [46]:
employee_dict['mtx']
Out[46]:
array([[560.33148363,   0.        , 651.26264911],
       [  0.        , 561.3767079 , 499.06540191],
       [  0.        ,   0.        ,   1.        ]])
In [47]:
np.shape(employee_dict['objpoints'])
Out[47]:
(35, 48, 3)
In [48]:
np.shape(employee_dict['imgpoints'])
Out[48]:
(35, 48, 1, 2)
In [ ]: